import json
import os
import re
import glob
import time
from tqdm import tqdm

# 清理文本，移除多余的空白字符
def clean_text(text):
    if text:
        # 替换连续的空白字符（空格、制表符、换行符等）为单个空格
        text = re.sub(r'\s+', ' ', text)
        # 去除首尾空白
        text = text.strip()
    return text

def process_file(input_file_path, output_dir):
    """处理单个JSON文件：清理文本并保存"""
    print(f"处理文件: {input_file_path}")
    
    filename = os.path.basename(input_file_path)
    output_file_path = os.path.join(output_dir, f"clean_{filename}")
    
    try:
        with open(input_file_path, 'r', encoding='utf-8') as f:
            try:
                # 加载JSON数据
                all_data = json.load(f)
                
                # 处理数据
                cleaned_data = []
                if isinstance(all_data, list):
                    for data in tqdm(all_data, desc=f"清理 {filename}"):
                        try:
                            # 清理文本字段
                            if 'title' in data:
                                data['title'] = clean_text(data['title'])
                            if 'text' in data:
                                data['text'] = clean_text(data['text'])
                            if 'meta_keywords' in data:
                                data['meta_keywords'] = clean_text(data['meta_keywords'])
                            if 'meta_description' in data:
                                data['meta_description'] = clean_text(data['meta_description'])
                            if 'paragraphs' in data and isinstance(data['paragraphs'], list):
                                data['paragraphs'] = [clean_text(p) for p in data['paragraphs'] if p]
                            
                            cleaned_data.append(data)
                        except Exception as e:
                            print(f"处理数据项出错: {e}")
                            # 保留原始数据
                            cleaned_data.append(data)
                else:
                    # 单个对象的情况
                    data = all_data
                    # 清理文本字段
                    if 'title' in data:
                        data['title'] = clean_text(data['title'])
                    if 'text' in data:
                        data['text'] = clean_text(data['text'])
                    if 'meta_keywords' in data:
                        data['meta_keywords'] = clean_text(data['meta_keywords'])
                    if 'meta_description' in data:
                        data['meta_description'] = clean_text(data['meta_description'])
                    if 'paragraphs' in data and isinstance(data['paragraphs'], list):
                        data['paragraphs'] = [clean_text(p) for p in data['paragraphs'] if p]
                    
                    cleaned_data = data
                
                # 保存清理后的数据
                with open(output_file_path, 'w', encoding='utf-8') as out_f:
                    json.dump(cleaned_data, out_f, ensure_ascii=False, indent=2)
                
                print(f"已保存清理后的文件: {output_file_path}")
                return True
            
            except json.JSONDecodeError as e:
                print(f"JSON解析错误: {e}, 文件: {input_file_path}")
                return False
    except Exception as e:
        print(f"处理文件出错: {e}, 文件: {input_file_path}")
        return False

def main():
    # 源数据目录
    data_dir = "crawler/data"
    # 清理后的数据目录
    clean_data_dir = "crawler/cleaned_data"
    
    # 创建输出目录
    os.makedirs(clean_data_dir, exist_ok=True)
    
    # 获取所有JSON文件
    json_files = glob.glob(os.path.join(data_dir, "*.json"))
    print(f"发现 {len(json_files)} 个JSON文件")
    
    # 处理所有文件
    start_time = time.time()
    successful_files = 0
    
    for file_path in json_files:
        if process_file(file_path, clean_data_dir):
            successful_files += 1
    
    end_time = time.time()
    elapsed_time = end_time - start_time
    
    print(f"\n处理完成! 成功处理 {successful_files}/{len(json_files)} 个文件")
    print(f"总耗时: {elapsed_time:.2f} 秒")

if __name__ == "__main__":
    main() 